{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb  # 模型\n",
    "import pandas as pd  # 数据处理包\n",
    "import numpy as np  # 数据处理包\n",
    "from sklearn import metrics  # 混淆矩阵\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV, train_test_split  # 分层五折验证包、寻找最优参函数、切分数据\n",
    "from sklearn.metrics import accuracy_score, roc_curve, auc, confusion_matrix  # 准确率、roc计算、auc计算、混淆矩阵\n",
    "import matplotlib.pyplot as plt  # 图形处理包\n",
    "import itertools  # 处理混淆矩阵\n",
    "import gc  # 处理缓存，有兴趣的可以搜搜怎么使用\n",
    "import warnings  # 忽略普通警告，不打印太多东西\n",
    "warnings.filterwarnings('ignore')\n",
    "plt.rcParams['font.sans-serif']=['SimHei']  # 让图形可以显示中文\n",
    "plt.rcParams['axes.unicode_minus']=False\n",
    "\n",
    "\n",
    "def preprocessing(df_train, df_pre, feats_list=0, label='label', id_1='id'):\n",
    "    \"\"\"\n",
    "    功能:将训练集和预测集做处理，并输出\n",
    "    df_train: 未处理过的训练集（df型/有label）\n",
    "    df_pre: 未处理过的预测集（df型/有label）\n",
    "    feats_list: 特征表，默认使用全部特征，可传入list\n",
    "    label: 标签，判断0/1\n",
    "    id: id\n",
    "    return: \n",
    "        dt_train: 处理过的训练集\n",
    "        df_pre: 处理过的预测集\n",
    "        X: 训练数据X（df型/无label）\n",
    "        y: 训练数据y（df型/label）\n",
    "        X_test_v1: 预测数据X（df型/无label）\n",
    "        y_test_v1: 预测数据y（df型/label）\n",
    "    \"\"\"\n",
    "    print(\"=============开始处理数据集===============\")\n",
    "    print(\"未处理训练集数据大小：\", df_train.shape)\n",
    "    print(\"未处理预测集数据大小：\", df_pre.shape)\n",
    "    # 和if不同，try的代码即使错了，也不会报错中断代码\n",
    "    try:\n",
    "        df_train = df_train.dropna(subset=['A字段']).reset_index(drop=True)  # 删掉指定列为null值的行\n",
    "        df_train = df_train[df_train['B字段'] >= 0].reset_index(drop=True)  # 去掉指定列非数值的行\n",
    "        df_pre = df_pre.dropna(subset=['A字段']).reset_index(drop=True)  # 预测集做同样的操作\n",
    "        df_pre = df_pre[df_pre['B字段'] >= 0].reset_index(drop=True)\n",
    "    except KeyError:\n",
    "        print(\"数据集里没有相应字段\")\n",
    "        \n",
    "    print(\"处理后训练集数据大小：\", df_train.shape)\n",
    "    print(\"处理后预测集数据大小：\", df_pre.shape)    \n",
    "    print(\"=============数据集处理完成===============\")\n",
    "    \n",
    "    print(\"==============开始切分数据=================\")\n",
    "    if feats_list==0:\n",
    "        print(\"使用全部特征\")\n",
    "        X = df_train[df_train.columns.drop([id_1, label])]\n",
    "        X_test_v1 = df_pre[df_pre.columns.drop([id_1, label])]\n",
    "    elif type(feats_list) == list:\n",
    "        print(\"使用列表特征，长度为：\", len(feats_list))\n",
    "        X = df_train[feats_list]\n",
    "        X_test_v1 = df_pre[feats_list]\n",
    "    else:\n",
    "        print(\"feats_list输入有误\")\n",
    "    y = df_train[label]\n",
    "    y_test_v1 = df_pre[label]\n",
    "    X = X.fillna(0)  # null值补0，这是经过模型做出来的最好处理方式\n",
    "    X_test_v1 = X_test_v1.fillna(0)\n",
    "    print(\"训练集正负样本情况：\")\n",
    "    print(pd.value_counts(df_train[label]))\n",
    "    print(\"预测集正负样本情况：\")\n",
    "    print(pd.value_counts(df_pre[label]))\n",
    "    print(\"==============数据切分完成=================\")\n",
    "    return df_train, df_pre, X, y, X_test_v1, y_test_v1\n",
    "\n",
    "\n",
    "def train_5_cross(df_pre, X,y, X_test_v1,y_test_v1, thresholds=0.45, id_1='id', csv_name=0):\n",
    "    \"\"\"\n",
    "    功能: 五折训练并输出名单\n",
    "    df_pre：原始预测数据\n",
    "    X: 训练数据X（无标签/df型）\n",
    "    y: 训练数据y（标签/df型）\n",
    "    X_test_v1: 预测数据X（无标签/df型）\n",
    "    y_test_v1: 预测数据y（无标签/df型）\n",
    "    thresholds: 阈值选择，默认0.45高精确率\n",
    "    csv_name: 保存csv的名称，默认不保存\n",
    "    returen:\n",
    "        模型，客户名单及情况\n",
    "    \"\"\"\n",
    "    vali_auc_num=0  # 验证集AUC\n",
    "    vali_recall_num=0  # 验证集召回率\n",
    "    vali_precision_num=0  # 验证集精确率\n",
    "    test_auc_num=0  # 预测集AUC\n",
    "    test_recall_num=0  # 预测集召回率\n",
    "    test_precision_num=0  # 预测集精确率\n",
    "    y_pred_input = np.zeros(len(X_test_v1))  # 相应大小的零矩阵\n",
    "    print(\"=============开始训练================\")\n",
    "    folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1234)  # 分层采样, n_splits为几折\n",
    "    for fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y)):\n",
    "        print(\"第 {} 次训练...\".format(fold_+1))\n",
    "        train_x, trai_y = X.loc[trn_idx], y.loc[trn_idx]\n",
    "        vali_x, vali_y = X.loc[val_idx], y.loc[val_idx]\n",
    "        \n",
    "        # 以下为调过参的lgb模型\n",
    "        clf = lgb.LGBMClassifier(max_depth=20, min_data_in_bin=5, max_bin=200,\n",
    "                                min_child_samples=90, num_leaves=20, n_estimators=20000,\n",
    "                                objective='binary', boosting_type='gbdt', learning_rate=0.02,\n",
    "                                lambda_l2=5)\n",
    "        clf.fit(train_x, trai_y, eval_set=[(train_x, trai_y), (vali_x, vali_y)], verbose=0,\n",
    "               early_stopping_rounds=100, eval_metric='f1')\n",
    "        \n",
    "        # 不懂的去GitHub看搜LightGBM的参数解释\n",
    "        \n",
    "        # ===============验证集AUC操作===================\n",
    "        y_prb = clf.predict_proba(vali_x)[:,1]  # 获取预测概率\n",
    "        # fpr:在实际为正的样本中，被正确判断为正的比例。tpr:在实际为负的样本中，被正确判断为负的比例。thres为阈值\n",
    "        fpr, tpr, thres = roc_curve(vali_y, y_prb)\n",
    "        vali_roc_auc = auc(fpr, tpr)  # 获取验证集auc\n",
    "        vali_auc_num += vali_roc_auc  # 将本次auc加入总值里\n",
    "        print(\"vali auc = {0:.4}\".format(vali_roc_auc))  # 本次auc的值\n",
    "        # ===============预测集AUC操作===================\n",
    "        y_prb_test = clf.predict_proba(X_test_v1)[:,1]  # 获取预测概率\n",
    "        fpr, tpr, thres = roc_curve(y_test_v1, y_prb_test)\n",
    "        test_roc_auc = auc(fpr, tpr)\n",
    "        test_auc_num += test_roc_auc\n",
    "        print(\"test auc = {0:.4}\".format(test_roc_auc))\n",
    "        \n",
    "        # ===============验证metric操作===================\n",
    "        y_pre_proba = clf.predict_proba(vali_x.values)\n",
    "        y_predictions = y_pre_proba[:, 1]>thresholds  # 取阈值多少以上的为True\n",
    "        cnf_matrix = confusion_matrix(vali_y, y_predictions)  # 建立矩阵\n",
    "        np.set_printoptions(precision=2)  # 控制在两位数\n",
    "        vali_recall = '{0:.3f}'.format(cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))  # 召回率\n",
    "        vali_precision = '{0:.3f}'.format(cnf_matrix[1,1]/(cnf_matrix[0,1]+cnf_matrix[1,1]))  # 精确率\n",
    "        print(\"vali_metric: \", vali_recall, vali_precision)\n",
    "        vali_recall_num += float(vali_recall)  # 将本次召回率加入总值里\n",
    "        vali_precision_num += float(vali_precision)  # 将本次精确率加入总值里\n",
    "        # ===============预测metric操作===================\n",
    "        y_pre_proba_test = clf.predict_proba(X_test_v1.values)\n",
    "        y_predictions_test = y_pre_proba_test[:, 1]>thresholds  # 取阈值多少以上的为True\n",
    "        cnf_matrix_test = confusion_matrix(y_test_v1, y_predictions_test)  # 建立矩阵\n",
    "        np.set_printoptions(precision=2)  # 控制在两位数\n",
    "        test_recall = '{0:.3f}'.format(cnf_matrix_test[1,1]/(cnf_matrix_test[1,0]+cnf_matrix_test[1,1]))  # 召回率\n",
    "        test_precision = '{0:.3f}'.format(cnf_matrix_test[1,1]/(cnf_matrix_test[0,1]+cnf_matrix_test[1,1]))  # 精确率\n",
    "        print(\"test_metric: \", test_recall, test_precision)\n",
    "        test_recall_num += float(test_recall)  # 将本次召回率加入总值里\n",
    "        test_precision_num += float(test_precision)  # 将本次精确率加入总值里\n",
    "        y_pred_input += y_pre_proba_test[:, 1]  # 将每次的预测的结果写入数组中\n",
    "        \n",
    "    print(\"5折泛化，验证集AUC：{0:.3f}\".format(vali_auc_num/5))  # 前面是做了5次相加，所以这次要除以5\n",
    "    print(\"5折泛化，预测集AUC：{0:.3f}\".format(test_auc_num/5))\n",
    "    \n",
    "    print(\"5折泛化，验证集recall：{0:.3f}\".format(vali_recall_num/5))\n",
    "    print(\"5折泛化，验证集precision：{0:.3f}\".format(vali_recall_num/5))\n",
    "    \n",
    "    print(\"5折泛化，预测集recall：{0:.3f}\".format(test_recall_num/5))\n",
    "    print(\"5折泛化，预测集precision：{0:.3f}\".format(test_recall_num/5))\n",
    "    \n",
    "    print(\"================开始输出名单==================\")\n",
    "    y_pred_input_end = y_pred_input / 5  # 前面是做了5次相加，所以这次要除以5\n",
    "    y_pred_input_precision = y_pred_input_end > thresholds  # 获取高精确率的标签\n",
    "    submission = pd.DataFrame({\"id\": df_pre[id_1],\n",
    "                              \"概率\": y_pred_input_end,\n",
    "                              \"高精确\": y_pred_input_precision})\n",
    "    if csv_name != 0:\n",
    "        submission.to_csv(\"%s预测名单.csv\" % csv_name, index=False)  # 保存\n",
    "    print(\"================输出名单名单==================\")\n",
    "    print(submission.head(5))\n",
    "    return clf\n",
    "    \n",
    "    \n",
    "def auc_plot(y, y_prob, thres=0.5, png_savename=0):\n",
    "    \"\"\"\n",
    "    why: 能够知道模型的效果，AUC越高，则模型分辨正负样本的能力越好。\n",
    "    功能: 画出AUC图\n",
    "    y: 真实结果（标签/df型）\n",
    "    y_prob：预测概率（浮点数）\n",
    "    png_savename: 保存图片的名字，默认不保存\n",
    "    return: AUC图\n",
    "    \"\"\"\n",
    "    y_prediction = y_prob > thres  # 输出预测的结果\n",
    "    fpr, tpr, thres = roc_curve(y, y_prob)\n",
    "    roc_auc = auc(fpr, tpr)  # 计算AUC\n",
    "    # 画出AUC\n",
    "    plt.plot(fpr, tpr, label=\"AUC = {0:.4f}\".format(roc_auc), ms=100)\n",
    "    plt.title(\"ROC曲线\", fontsize=20)\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.plot([0, 1], [0, 1], 'r--')\n",
    "    plt.xlabel('误判率', fontsize=15)\n",
    "    plt.ylabel('命中率', fontsize=15)\n",
    "    if png_savename != 0:\n",
    "        plt.savefig(\"%s_AUC图.png\" % png_savename, dpi=300)  # 保存AUC图, dpi清晰度，越大越清晰\n",
    "    plt.show()\n",
    "    print(\"Accuracy: {0:.2f}\".format(accuracy_score(y, y_prediction)))  # 准确率\n",
    "    \n",
    "    \n",
    "def metrics_ks(y, y_predicted):\n",
    "    \"\"\"\n",
    "    功能: 计算模型性能指标：ks， 找到最佳threshold值\n",
    "    y: 数据y（标签/df型）\n",
    "    y_predicted: 概率值， 公式为：= clf.predict_proba(X)[:, 1]\n",
    "    return:\n",
    "        ks值\n",
    "        thres_ks值\n",
    "    \"\"\"\n",
    "    fpr, tpr, thres = metrics.roc_curve(y, y_predicted, pos_label=1)\n",
    "    ks = abs(fpr - tpr).max()  # abs:返回数字绝对值的函数\n",
    "    tmp = abs(fpr - tpr)\n",
    "    index_ks = np.where(tmp=ks)  # np.where: 返回符合条件的下标函数\n",
    "    thres_ks = thres[index_ks]\n",
    "    return ks, thres_ks\n",
    "    \n",
    "\n",
    "def plot_confusion_matrix(cm, classes, title='Confusion matrix', cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    混淆矩阵画图函数\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "    \n",
    "    thresh = cm.max()/2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i,j], horizontalalignment='center', color='white' if cm[i, j]> thresh else 'black')\n",
    "        plt.tight_layout()\n",
    "        plt.ylabel(\"True label\")\n",
    "        plt.xlabel(\"Predicted label\")\n",
    "        \n",
    "        \n",
    "def metrics_plot(y, y_prob, thres=0.45, png_savename=0):\n",
    "    \"\"\"\n",
    "    why: 能选择是召回率高，还是精确率高，也能从一定层面看出模型的效果。\n",
    "    功能: 画出混淆矩阵图\n",
    "    y: 真实值y（标签/df型）\n",
    "    y_prob：预测概率\n",
    "    thres: 阈值，多少以上为预测正确\n",
    "    png_savename: 保存图片的名字，默认不保存\n",
    "    return: 输出混淆矩阵图\n",
    "    \"\"\"\n",
    "    y_prediction = y_prob > thres  # 多少以上的概率判定为正\n",
    "    cnf_matrix = confusion_matrix(y, y_prediction)  # 形成混淆矩阵\n",
    "    np.set_printoptions(precision=2)  # 设置浮点精度\n",
    "    vali_recall = '{0:.3f}'.format(cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))  # 召回率\n",
    "    vali_precision = '{0:.3f}'.format(cnf_matrix[1,1]/(cnf_matrix[0,1]+cnf_matrix[1,1]))  # 精确率\n",
    "    class_names = [0,1]\n",
    "    plot_confusion_matrix(cnf_matrix, classes=class_names, title='召回率=%s%% \\n 精确率=%s%%' %('{0:.1f}'.format(float(vali_recall)*100),\n",
    "                                                                                         '{0:.1f}'.format(float(vali_precision)*100)))\n",
    "    if png_savename!=0:\n",
    "        plt.savefig(\"%s_混淆矩阵.png\" % png_savename,dpi=300)  # 保存混淆矩阵图\n",
    "        \n",
    "\n",
    "def importance_plt(X, clf, png_savename=0):\n",
    "    \"\"\"\n",
    "    why: 能看出哪个特征更重要，继而对特征做相关衍生，也可以讲特征使用次数为0的特征去掉，防止冗余\n",
    "    功能:打印特征重要图\n",
    "    X: 数据X（无标签/df型）\n",
    "    clf: 已训练过的最佳lgb模型\n",
    "    png_savename: 保存图片的名字，默认不保存\n",
    "    return: 打印出特征重要性图\n",
    "    \"\"\"\n",
    "    feats_list = X.columns.values  # 获取全部特征\n",
    "    feats_importance = clf.feature_importances_  # 获取特征使用次数\n",
    "    sorted_idx = np.argsort(feats_importance)\n",
    "    \n",
    "    plt.figure(figsize=(10, 55))\n",
    "    # 下面是画图操作\n",
    "    plt.barh(range(len(sorted_idx)), feats_importance[sorted_idx])\n",
    "    plt.yticks(range(len(sorted_idx)), feats_list[sorted_idx])\n",
    "    plt.xlabel(\"Importance\")\n",
    "    plt.title(\"Feature importances\")\n",
    "    if png_savename:\n",
    "        plt.savefig(\"特征重要性.png\", dpi=500, bbox_inches='tight')  # 由于特征过多图片过大，所以需要这些处理才能让图片全部保存下来\n",
    "    plt.show()\n",
    "    \n",
    "    \n",
    "def just_num_leaves(X, y, start_num=10, end_num=101, step=10):\n",
    "    \"\"\"\n",
    "    功能: 找到最优num_leaves参数，以此类推找出全部的最优参\n",
    "    X: 数据X（无标签/df型）\n",
    "    y: 数据y（标签/df型）\n",
    "    start_num: 开始值\n",
    "    end_num: 最大值\n",
    "    step: 步数\n",
    "    return: 最佳num_leaves\n",
    "    \"\"\"\n",
    "    param_dic = {'num_leaves': range(start_num, end_num, step)}\n",
    "    gscv = GridSearchCV(estimator=lgb.LGBMClassifier(max_depth=20, min_data_in_bin=5, max_bin=200,\n",
    "                                                     min_child_samples=90, n_estimators=20000,\n",
    "                                                     objective='binary', boosting_type='gbdt', learning_rate=0.02,\n",
    "                                                     lambda_l2=5),\n",
    "                       param_grid=param_dic, scoring='f1', cv=5)\n",
    "    gscv.fit(X, y)\n",
    "    print(\"best_params:{0}\".format(gscv.best_params_))\n",
    "    print(\"best_score:{0}\".format(gscv.best_score_))\n",
    "    \n",
    "    \n",
    "def train_2_cross(df_pre, X,y, X_test_v1,y_test_v1, thresholds=0.45, id_1='id', csv_name=0):\n",
    "    \"\"\"\n",
    "    功能:切分一次训练，输出名单\n",
    "    X: 训练数据X（无标签/df型）\n",
    "    y: 训练数据y（标签/df型）\n",
    "    X_test_v1: 预测数据X（无标签/df型）\n",
    "    y_test_v1: 预测数据y（无标签/df型）\n",
    "    thresholds: 阈值选择，默认0.45高精确率\n",
    "    csv_name: 保存csv的名称，默认不保存\n",
    "    returen:\n",
    "        客户名单及情况\n",
    "        clf: 已训练好的模型\n",
    "    \"\"\"\n",
    "    y_pred_input = np.zeros(len(X_test_v1))  # 相应大小的零矩阵\n",
    "    train_x, vali_x, train_y,vali_y = train_test_split(X, y, test_size=0.33, random_state=1234)\n",
    "    clf = lgb.LGBMClassifier(max_depth=20, min_data_in_bin=5, max_bin=200,\n",
    "                            min_child_samples=90, num_leaves=20, n_estimators=20000,\n",
    "                            objective='binary', boosting_type='gbdt', learning_rate=0.02,\n",
    "                            lambda_l2=5)\n",
    "    clf.fit(train_x, train_y, eval_set=[(train_x, train_y), (vali_x, vali_y)], verbose=0,\n",
    "           early_stopping_rounds=100, eval_metric='f1')\n",
    "    # 这里的参数不懂的去GitHub搜LightGBM的参数解释\n",
    "\n",
    "    # ===============验证集AUC操作===================\n",
    "    y_prb = clf.predict_proba(vali_x)[:,1]  # 获取预测概率\n",
    "    # fpr:在实际为正的样本中，被正确判断为正的比例。tpr:在实际为负的样本中，被正确判断为负的比例。thres为阈值\n",
    "    fpr, tpr, thres = roc_curve(vali_y, y_prb)\n",
    "    vali_roc_auc = auc(fpr, tpr)  # 获取验证集auc\n",
    "    print(\"vali auc = {0:.4}\".format(vali_roc_auc))  # 本次auc的值\n",
    "    # ===============预测集AUC操作===================\n",
    "    y_prb_test = clf.predict_proba(X_test_v1)[:,1]  # 获取预测概率\n",
    "    fpr, tpr, thres = roc_curve(y_test_v1, y_prb_test)\n",
    "    test_roc_auc = auc(fpr, tpr)\n",
    "    print(\"test auc = {0:.4}\".format(test_roc_auc))\n",
    "    \n",
    "    # ===============训练metric操作===================\n",
    "    y_pre_proba = clf.predict_proba(vali_x.values)\n",
    "    y_predictions = y_pre_proba[:, 1]>thresholds  # 取阈值多少以上的为True\n",
    "    cnf_matrix = confusion_matrix(vali_y, y_predictions)  # 建立矩阵\n",
    "    np.set_printoptions(precision=2)  # 控制在两位数\n",
    "    vali_recall = '{0:.3f}'.format(cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))  # 召回率\n",
    "    vali_precision = '{0:.3f}'.format(cnf_matrix[1,1]/(cnf_matrix[0,1]+cnf_matrix[1,1]))  # 精确率\n",
    "    print(\"vali_metric: \", vali_recall, vali_precision)\n",
    "    # ===============预测metric操作===================\n",
    "    y_pre_proba_test = clf.predict_proba(X_test_v1.values)\n",
    "    y_predictions_test = y_pre_proba_test[:, 1]>thresholds  # 取阈值多少以上的为True\n",
    "    cnf_matrix_test = confusion_matrix(y_test_v1, y_predictions_test)  # 建立矩阵\n",
    "    np.set_printoptions(precision=2)  # 控制在两位数\n",
    "    test_recall = '{0:.3f}'.format(cnf_matrix_test[1,1]/(cnf_matrix_test[1,0]+cnf_matrix_test[1,1]))  # 召回率\n",
    "    test_precision = '{0:.3f}'.format(cnf_matrix_test[1,1]/(cnf_matrix_test[0,1]+cnf_matrix_test[1,1]))  # 精确率\n",
    "    print(\"test_metric: \", test_recall, test_precision)\n",
    "\n",
    "    print(\"================开始输出名单==================\")\n",
    "    y_pred_input_precision = y_pre_proba_test[:, 1] > thresholds  # 获取高精确率的标签\n",
    "    submission = pd.DataFrame({\"id\": df_pre[id_1],\n",
    "                              \"概率\": y_pre_proba_test[:, 1],\n",
    "                              \"高精确\": y_pred_input_precision})\n",
    "    if csv_name != 0:\n",
    "        submission.to_csv(\"%s预测名单.csv\" % csv_name, index=False)  # 保存\n",
    "    print(\"================输出名单名单==================\")\n",
    "    print(submission.head(5))\n",
    "    return clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('../data/train.csv')\n",
    "test_data = pd.read_csv('../data/train.csv')  # 这里读train，因为有label比较对错，可以改成test并修改相应的数据处理和训练函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>date</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>...</th>\n",
       "      <th>f288</th>\n",
       "      <th>f289</th>\n",
       "      <th>f290</th>\n",
       "      <th>f291</th>\n",
       "      <th>f292</th>\n",
       "      <th>f293</th>\n",
       "      <th>f294</th>\n",
       "      <th>f295</th>\n",
       "      <th>f296</th>\n",
       "      <th>f297</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cb3dff566c2e804ee5842a50d1641f934e2a3b5f630c4d...</td>\n",
       "      <td>0</td>\n",
       "      <td>20171026</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>100805.0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d59c75a60aaf0ad51d05624de6a78c32ab87e07a0847bc...</td>\n",
       "      <td>0</td>\n",
       "      <td>20170930</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100804.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73af2a7c4d2991c898010c2c8fd00b3d81d3f8f6a7fca...</td>\n",
       "      <td>0</td>\n",
       "      <td>20171028</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aebb59c3288fea434e60c94d732abff99ac8e1f2c503fa...</td>\n",
       "      <td>0</td>\n",
       "      <td>20171021</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5d2d91c0023379976e97fbce2f30217fff35690c681305...</td>\n",
       "      <td>0</td>\n",
       "      <td>20170907</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>100805.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>301.0</td>\n",
       "      <td>301.0</td>\n",
       "      <td>302.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>274.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>251.0</td>\n",
       "      <td>251.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>227.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  id  label      date  f1  f2  \\\n",
       "0  cb3dff566c2e804ee5842a50d1641f934e2a3b5f630c4d...      0  20171026   0   1   \n",
       "1  d59c75a60aaf0ad51d05624de6a78c32ab87e07a0847bc...      0  20170930   0   0   \n",
       "2  e73af2a7c4d2991c898010c2c8fd00b3d81d3f8f6a7fca...      0  20171028   1   0   \n",
       "3  aebb59c3288fea434e60c94d732abff99ac8e1f2c503fa...      0  20171021   1   0   \n",
       "4  5d2d91c0023379976e97fbce2f30217fff35690c681305...      0  20170907   1   0   \n",
       "\n",
       "   f3  f4        f5  f6  f7  ...   f288   f289   f290   f291   f292   f293  \\\n",
       "0   1   0  100805.0   1   4  ...    2.0    1.0    1.0    1.0    2.0    1.0   \n",
       "1   1   1  100804.0   1   3  ...    2.0    3.0    4.0    1.0    1.0    1.0   \n",
       "2   2   0       NaN   2   2  ...    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3   1   0       NaN   1   5  ...    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4   1   0  100805.0   0   5  ...  301.0  301.0  302.0  126.0  274.0  250.0   \n",
       "\n",
       "    f294   f295   f296   f297  \n",
       "0    1.0    1.0    1.0    2.0  \n",
       "1    2.0    2.0    1.0    1.0  \n",
       "2    NaN    NaN    NaN    NaN  \n",
       "3    NaN    NaN    NaN    NaN  \n",
       "4  251.0  251.0  109.0  227.0  \n",
       "\n",
       "[5 rows x 300 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>date</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>...</th>\n",
       "      <th>f288</th>\n",
       "      <th>f289</th>\n",
       "      <th>f290</th>\n",
       "      <th>f291</th>\n",
       "      <th>f292</th>\n",
       "      <th>f293</th>\n",
       "      <th>f294</th>\n",
       "      <th>f295</th>\n",
       "      <th>f296</th>\n",
       "      <th>f297</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cb3dff566c2e804ee5842a50d1641f934e2a3b5f630c4d...</td>\n",
       "      <td>0</td>\n",
       "      <td>20171026</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>100805.0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d59c75a60aaf0ad51d05624de6a78c32ab87e07a0847bc...</td>\n",
       "      <td>0</td>\n",
       "      <td>20170930</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100804.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73af2a7c4d2991c898010c2c8fd00b3d81d3f8f6a7fca...</td>\n",
       "      <td>0</td>\n",
       "      <td>20171028</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aebb59c3288fea434e60c94d732abff99ac8e1f2c503fa...</td>\n",
       "      <td>0</td>\n",
       "      <td>20171021</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5d2d91c0023379976e97fbce2f30217fff35690c681305...</td>\n",
       "      <td>0</td>\n",
       "      <td>20170907</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>100805.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>301.0</td>\n",
       "      <td>301.0</td>\n",
       "      <td>302.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>274.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>251.0</td>\n",
       "      <td>251.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>227.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  id  label      date  f1  f2  \\\n",
       "0  cb3dff566c2e804ee5842a50d1641f934e2a3b5f630c4d...      0  20171026   0   1   \n",
       "1  d59c75a60aaf0ad51d05624de6a78c32ab87e07a0847bc...      0  20170930   0   0   \n",
       "2  e73af2a7c4d2991c898010c2c8fd00b3d81d3f8f6a7fca...      0  20171028   1   0   \n",
       "3  aebb59c3288fea434e60c94d732abff99ac8e1f2c503fa...      0  20171021   1   0   \n",
       "4  5d2d91c0023379976e97fbce2f30217fff35690c681305...      0  20170907   1   0   \n",
       "\n",
       "   f3  f4        f5  f6  f7  ...   f288   f289   f290   f291   f292   f293  \\\n",
       "0   1   0  100805.0   1   4  ...    2.0    1.0    1.0    1.0    2.0    1.0   \n",
       "1   1   1  100804.0   1   3  ...    2.0    3.0    4.0    1.0    1.0    1.0   \n",
       "2   2   0       NaN   2   2  ...    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3   1   0       NaN   1   5  ...    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4   1   0  100805.0   0   5  ...  301.0  301.0  302.0  126.0  274.0  250.0   \n",
       "\n",
       "    f294   f295   f296   f297  \n",
       "0    1.0    1.0    1.0    2.0  \n",
       "1    2.0    2.0    1.0    1.0  \n",
       "2    NaN    NaN    NaN    NaN  \n",
       "3    NaN    NaN    NaN    NaN  \n",
       "4  251.0  251.0  109.0  227.0  \n",
       "\n",
       "[5 rows x 300 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=============开始处理数据集===============\n",
      "未处理训练集数据大小： (100000, 300)\n",
      "未处理预测集数据大小： (100000, 300)\n",
      "数据集里没有相应字段\n",
      "处理后训练集数据大小： (100000, 300)\n",
      "处理后预测集数据大小： (100000, 300)\n",
      "=============数据集处理完成===============\n",
      "==============开始切分数据=================\n",
      "使用全部特征\n",
      "训练集正负样本情况：\n",
      "0    98145\n",
      "1     1855\n",
      "Name: label, dtype: int64\n",
      "预测集正负样本情况：\n",
      "0    98145\n",
      "1     1855\n",
      "Name: label, dtype: int64\n",
      "==============数据切分完成=================\n"
     ]
    }
   ],
   "source": [
    "df_train, df_pre, X, y, X_test_v1, y_test_v1 = preprocessing(train_data, test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=============开始训练================\n",
      "第 1 次训练...\n",
      "vali auc = 0.9787\n",
      "test auc = 0.9942\n",
      "vali_metric:  0.437 0.645\n",
      "test_metric:  0.700 0.861\n",
      "第 2 次训练...\n",
      "vali auc = 0.98\n",
      "test auc = 0.9939\n",
      "vali_metric:  0.447 0.654\n",
      "test_metric:  0.678 0.850\n",
      "第 3 次训练...\n",
      "vali auc = 0.9784\n",
      "test auc = 0.9936\n",
      "vali_metric:  0.396 0.590\n",
      "test_metric:  0.662 0.842\n",
      "第 4 次训练...\n",
      "vali auc = 0.9761\n",
      "test auc = 0.993\n",
      "vali_metric:  0.407 0.583\n",
      "test_metric:  0.653 0.833\n",
      "第 5 次训练...\n",
      "vali auc = 0.9811\n",
      "test auc = 0.9932\n",
      "vali_metric:  0.407 0.592\n",
      "test_metric:  0.635 0.824\n",
      "5折泛化，验证集AUC：0.979\n",
      "5折泛化，预测集AUC：0.994\n",
      "5折泛化，验证集recall：0.419\n",
      "5折泛化，验证集precision：0.419\n",
      "5折泛化，预测集recall：0.666\n",
      "5折泛化，预测集precision：0.666\n",
      "================开始输出名单==================\n",
      "================输出名单名单==================\n",
      "                                                  id        概率    高精确\n",
      "0  cb3dff566c2e804ee5842a50d1641f934e2a3b5f630c4d...  0.000410  False\n",
      "1  d59c75a60aaf0ad51d05624de6a78c32ab87e07a0847bc...  0.000523  False\n",
      "2  e73af2a7c4d2991c898010c2c8fd00b3d81d3f8f6a7fca...  0.004670  False\n",
      "3  aebb59c3288fea434e60c94d732abff99ac8e1f2c503fa...  0.003348  False\n",
      "4  5d2d91c0023379976e97fbce2f30217fff35690c681305...  0.000559  False\n"
     ]
    }
   ],
   "source": [
    "clf = train_5_cross(df_pre,X, y, X_test_v1, y_test_v1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vali auc = 0.9793\n",
      "test auc = 0.9912\n",
      "vali_metric:  0.421 0.591\n",
      "test_metric:  0.606 0.802\n",
      "================开始输出名单==================\n",
      "================输出名单名单==================\n",
      "                                                  id        概率    高精确\n",
      "0  cb3dff566c2e804ee5842a50d1641f934e2a3b5f630c4d...  0.000526  False\n",
      "1  d59c75a60aaf0ad51d05624de6a78c32ab87e07a0847bc...  0.000889  False\n",
      "2  e73af2a7c4d2991c898010c2c8fd00b3d81d3f8f6a7fca...  0.004408  False\n",
      "3  aebb59c3288fea434e60c94d732abff99ac8e1f2c503fa...  0.006726  False\n",
      "4  5d2d91c0023379976e97fbce2f30217fff35690c681305...  0.000456  False\n"
     ]
    }
   ],
   "source": [
    "clf = train_2_cross(df_pre,X,y, X_test_v1,y_test_v1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEfCAYAAABBHLFTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXiU1fXA8e/Jwr6DsgiKuyIKSlxZGhBFa5W6K661SqnWbr+61qVaW3fauqCgqFVBRVAUFxAXRFSsIIKCqIhBFkEgQCAQksyc3x/3DRnCJJkkM/POzHs+zzNPZt65M+8Zljm59773XFFVjDHGmCy/AzDGGJMaLCEYY4wBLCEYY4zxWEIwxhgDWEIwxhjjsYRgjDEGsIRgTMKISBMRaSMiORHHdhOR/USkcR3eRxIToTE7s4RgTOJcA2wA+kUcuxn4FjggljcQkS7AdyLymyjP3SkiBSJyuve4bcNDNkFmCcGYxNnm/VwfcWxrlZ+1uQLYG/gpynOFwF7AOhE5AvhRRC6pT6DGgCUEYxKpxPu5LeJYufczVNuLRaQlcDXwuKq+HKXJWu/nJlX9DHgReNjrVRhTZ5YQTEYRke4iolVuq0Vkuoj8vErbISLykYhsFZFCEXlORLpFec/2IjJGRH7y2s4UkeOqtGkuIgdF3oCO3tP7RBxr7x3bt2p7EWlU5dTXAGHgWnFuF5G9Ip6vSC5h7+fvgKOBpiLSu65/dsbk1N7EmLQ0G3gYyMUNuVwBvC4iV6jq4yJyGvAysAK4Fffl/QfgOBE5XFULAUSkNfCB9x4jgdXAr4F3RSRfVWd75+sLTKsmlmjH345ybD/gO++8hwPXA5er6gYROQs3/7APcGGV1x3vJbtDcfMV3YHXgV9UE48x0amq3eyWMTfcl6ECz1c5vhdu6GYxIMBSoAjoGtFmhPfav0ccu887dk7EsXZAMTAj4lh/r93fIo79zju2X8SxO7xj3SOOPeUd6+o9PhhYiUtEXXDJaB7wNZAH3A1MwSUnxc1HzAZGAb8FTgC6+f13Ybf0u9mQkQkEVV0GLMQljN64L9nXVHVFRLOngTLglwAikg1cAixT1QkR71UIvA/0FZEm3uEwDVcxBLQXLhH0wyWGpUAv4Ffe82fi5g+e8B6fo6rHqOqVqvqIqk5X1eUi0jQOMZkAsSEjEwjel3sX3G/Vh3mHv4xso6pbRWQZcKC3duAAoAPwTpS3nILrYTTFTR7XOklcB9OAa4GPgQLgDeBDVf3Ie34/ABHZF7gBaC0iubiexeHAQOB4YALwf3GMy2Q4SwgmUzUSkQ7e/U7AdUBn3JDN7t7x9VFetx73hdsG2Nc7trxqI1V9BHgkyuuzIhaiVfTAsyOOSQ3HKt5bgXsBROR3wG64+QS8BW37ejEe5b3k38CTuPmSQmABMAmYGiU+Y6plCcFkqtO9W4VtwL+A23C/VQOURnldxbGmQMuI18bqZu8WaXGUdktqeyMR6YmbL7hUVTd5h/fGDX2Vej/X4JLYJbhEsA3opKoL6xCzMYBddmoy1wzc5Or5uInXN1X1z6paTuX6gKqXeUYe20blvMAuvziJSFcR6e0N1UR6ENjfu93mHTs+4tgo71h+xLGXorz/mbihqgKgVER+LSI34OYZ+gEtVfUI4Hmgiaq+AawCxgP/E5FhUT6bMTWyHoLJVGtU9W3Y8eV6poj0VNUvcb9VgxuKqWo33MTyRioXfkVrdwfut/LdI9oBFKrqEu+867xjP0Qc2+AdW6aqBd6xzVHe/2zvvXfHDf+swa1WfqLivTyzgT+ISA/csNiJwOPAK1He05gaWQ/BBMHt3s+KoZz53s/DIhuJSAtgT+ArryfxOa6XcESU96y4jLUw7tE69+PWEeyD6wHsoaqHq+qaKu3ewPV43gcuBu5U1StUtThBcZkMZgnBZDxV/QK3CO0sEemhqvNxC8BOEZGuEU0vwfWaJ3uvW4+bmD1CRAZUNPJWCx8HvK+q8by6KDLmT1X1dVX93p1S+ojIlSIyskrTtrg5hA7Aw6p6YyLiMcFgCcEExe24q3lu8h7/AWgCvC8iV4vIHbjfypfhJp8r/B43JPSqiNwsIn8GZnnP3ZqIQEVkDxF5SEReFZEvgS3AHNxkeKuIdmcCc3FXF/0EDBSRZt5zfxGRGyPWSRhTO79XxtnNbvG8Uc1KZe+5l3HrBQ70Hp8IfIhb6VsIPEeUFb644aFxXpttwHvA0VXa9PXO25Bbp4j3mwOsA54FhkXGhZtUnuq95j3c5bT5uCuP3gKOAb7x4m3u99+J3dLnJqoaj7xiTKCJSD7uy/lh4KE6vvxO3OrobuqtnBaRjsA6rTIkJSK3AbfgEtPfgPsr2ojI2bgEUnGl1LWqem99Po8JJrvKyJj4qBiaWaeq0dYdVEtEKtYY7Cg1obtOHlf4B+7/7ShVXRn5hKq+KCJLcYliMW4IzJiYWQ/BGGMMYJPKxhhjPJYQjDHGAGk+h9ChQwft3r2732EYY0zamDt37jpVjbb6Pr0TQvfu3ZkzZ47fYRhjTNrwSrxHZUNGxhhjAEsIxhhjPJYQjDHGAJYQjDHGeCwhGGOMAZKcEESko4h8UMPzuSIyRUQ+FJHLkhmbMcYEXdISgoi0Bf4LNK+h2dXAXFXti6td37KGtsYYY+IomesQQsC51Ly1Xz5wvXd/JpCHqyBp4qwsFCasSmQpK1VQ1PvpSqNDxX13R9GItl759Irn8Z7XytdEbe+1LQ8rqzeVkCVV2ld5fdVzEXk8ahyRr6ls+/26YprmZkf986iIc5fjNZT6qu6p6l5T3TlqO0/1r6lbzDWdoq4x1+fPpaYXJePPMp6fv6Y3q/6zxCfmzt98waZDevF/Qw6qPoh6SlpCUNUiABGpqVlzoKKCYyHQsWoDERkODAfYc8894xtkEpSFwmwoLmXdllLKw2HKQkp5KExRSTk/btpGbnYWYVXC6v4BhcPufsWXd8VzYa9++TdrttCsUTahsBIKK+VhJaRKKKQsWbtlx3lDYaUsFCYUVn7cVFJDhMYkTk3//at7qrrvjJq+Sao7j9T0qmpfU7dz1HSehnz+Vts2c830xzj987e45fybYMjfq3+zekq1lcpbcCWANwEtvMc7UdUxwBiAvLy8lC7V+vnyjSxbX8yCFZt4/n8/UFyakN0WAejSugnZ2UK2CNlZQk5WFrnZWazdXMKR3duRk51FbpaQky1kZ2VRVFJGj85u8y2Ryn/A7n7lsch/wCKy4zmoaCc7XlPxRNXXy47Xe+eJOLa9PMzeHZp7r9/5NZHvLV6DnZ6LEudO540Sw+6tmpCdVbcvmHj/x6/2verxJVb9a6prX8N71fUc9fmQpn5efhmuvBrWroUbbuD2W/6akNOkWkKYi9sNaiLQC5jtbzh1t2LDVh56dwnPf7p8l+f67NWW/AN2o1GO+7Les10zcrKF3OwscrKExrnZdGrVhCxx/9myBLJEyBJBsiruu59Q+Tg7S+w/pzGZ6oUX4LzzoHdveP11OOKIhJ3Kt4QgIoOAHqoaubvUf4E3RKQ/0AP4xJfg6mh54Vae+qiAsbO+3+n4kEM6cnn/fejatintmjeicU708WtjjNmJKqxaBXvsAaefDqNGweWXQ25uQk+bchvkiEgXXC9hmqpuqqltXl6e+lncLhRWbnhpARPmrNhx7KRDOjHs6D05bt/25GTbMg9jTB0tWwa/+Q0sXAiLFkHL+F5sKSJzVTUv2nOpNmSEqq4CJvgdR03mLivk/a/X8sC7SwBo3TSXu888lCGHdLKhG2NM/YTD8MgjcP31rodw113QvKar9OMv5RJCKisLhblu0gJe+sxdCNWySQ4nHNyR+8/pZYnAGFN/GzfCqafCrFlw4okwejT4sNeLJYQYlYfCnDdmNnOXbeCAji144PzDOahTK7/DMsZkgtatoUsXeOopuPji+l2mFgeWEGKwZXs5Q/41k5Ubt3HGEXsw8pzefodkjEl38+bBn/4EzzwD3bq5q4l8ZrOetVBVbnzpC1Zu3Ma5ed0sGRhjGqakBG68EY48EhYvhoICvyPawXoINdheHiL/3hn8uKmEHp1bcfdZh/kdkjEmnc2a5S4f/fpr+NWv4P77oW1bv6PawRJCDa6duIAfN5VwSJdWjL/iGL/DMcaku6eegu3bYdo0N3mcYiwhVOOFT3/glc9XcVCnlrx2dT+7isgYUz/TpkHHjm6l8ciRkJUFLVr4HVVUNocQRWl5mNunLKJpbjb/vewoSwbGmLorLIRLLoGTToI773THWrVK2WQAlhCiemb2MopLQzxw/uF0bNXE73CMMelm4kQ4+GAYPx7++lf473/9jigmNmQUxbuL19ChRSNO6LFL9W1jjKnZ88/D+ee7InTTprmhojRhPYQqlhdu5cMl6znl0M5+h2KMSReqsMKraXbGGfDoo/DJJ2mVDMASwi6umTgfgAuP2cvnSIwxaaGgAIYMgWOOgaIiaNTIFafLSb8BGEsIEUJhZU7BBvIP3I39O9p2zsaYGoRC8MAD0LMnfPyxW2yWwhPGsUi/FJZA81dspDysnHpYF79DMcaksg0b4JRTXCI4+WQ3RJSGW/pWZT2ECM9+vAyA/gd08DkSY0xKa9PGVSN95hm3i1kGJAOwhLCTJWvdhvW7t7RLTY0xVcydC/37ww8/uGqk48fDhRf6Vpk0ESwhRFhT5MpUGGPMDtu2uU1rjj4avvsOlu+6X3qmsITg+XHTNtYUbeeknna5qTHGM3Mm9OoFd9/titEtWgR9+/odVcLYpLLn3cU/AZC3V+pUHjTG+Ozpp6G8HN5+G44/3u9oEs4SgmfaQrc6uecerf0OxRjjpzfegM6d4fDDXTG67Oyk723sFxsyAjZtLWPmN2vpu18HsrMyZ4LIGFMH69bBRRe5y0nvvtsda9UqMMkALCEA8NnyDQD8svcePkdijEk6Vbd9ZY8erg7RzTenTTG6eLMhI2Dp2mIAetgVRsYEz/PPw7Bh0KePmys4LLg7I1oPAfhu7RZaN81l95aN/Q7FGJMMqm49AcCZZ8KYMTB7dqCTAVhCAGDRqiL2272FbYRjTBAsXQqDB8Nxx1UWo7viirQsRhdvgU8IZaEwC1dt4vBubfwOxRiTSKEQ/Otfrhjdp5/CLbekfTG6eAt8SixYV0xZSDm4s80fGJOxNmxwReg++cRdRfToo9C1q99RpZzA9xDmr9gEwIGdrNy1MRmrTRvYbz8YNw6mTLFkUI3AJ4SVG7YBsH9H6zoak1E+/dSVmVi2zBWge/ZZdzWRzRVWK/AJYXXRNjq0aETjnGy/QzHGxMPWrXDNNW4Hs4KCyq0tTa0CnxB+KtrOblbu2pjMMGOGK0Z3331w+eUZX4wu3pI2qSwiY4EewOuqekeU59sC44Ddgbmq+ptkxLVy4za6tm2WjFMZYxJt3Di3xuDdd2HgQL+jSTtJ6SGIyBlAtqoeC+wjIvtHaXYRME5V84CWIpKXjNg2l5TTumluMk5ljEmE116Dzz5z90eOhAULLBnUU7KGjPKBCd79t4B+UdqsB3qKSBugG5CUXSjWbt5Oq6aBv/rWmPSzdq2bJD71VLj3XnesZUtoZj3++kpWQmgOrPTuFwIdo7SZBewF/B74ymu3CxEZLiJzRGTO2rVrGxTU5pIySkNhmjWyCWVj0oYqPPecK0Y3cSLcdltgi9HFW7ISwhagqXe/RTXnvRUYoaq3A4uBX0V7I1Udo6p5qpq32267NSiodVtKAejcumktLY0xKaOiGN2++8K8eW7FcaNGfkeVEZKVEOZSOUzUCyiI0qYtcKiIZANHA5rooArWuyqne7azLqYxKS0cdusJwBWjGzsWPvwQDjnE37gyTLISwmTgIhEZCZwDLBSRqlca3QmMATYB7YDnEh3UjxtLAGjX3H67MCZlLVnitq+MLEZ32WVuJzMTV0mZTVXVIhHJB04A7lHV1cD8Km3+ByQ13W/c5oaMura1ISNjUk4oBP/+t9uwJjcX7r/fTRqbhEna5TWquoHKK41Swo8bS2jZOMcuOzUm1WzYAEOGuPITp50Go0bBHrajYaIFeqXyxm1ltG/RyPZBMCZVqDd12KYNHHigm0CePNmSQZIEOiEUFm+3+QNjUsXs2XDssZXF6J55Bs4914rRJVHAE0KZJQRj/FZcDH/+s5s0XrkSVq3yO6LACnRCKNpWRiubPzDGP++8A4ce6nYyGzECFi50vQTji0DXbFhfvJ0OLRr7HYYxwfXCC24v4/ffhwED/I4m8AKbEDaXlFFSFqa9DRkZk1yvvuomifv0cZeS5uRAU7v0OxUEdsioqKQcwC45NSZZfvoJzjsPhg51iQDcugJLBikjsAlhQ7FblNammSUEYxJK1W1fefDB8PLL8Pe/WzG6FBXYIaOSshAAzRsH9o/AmOR47jm46CK3peXYsa5KqUlJgf023FrqEkKTXKuHYkzcVRSj23tvOPtsKC11ScHqD6W0wA4ZFZWUAdCqiQ0ZGRNX33wD+fluL+OiIleH6NJLLRmkgcAmhE3bvIRgu6UZEx/l5XDPPW6T+y++gH/+04rRpZnAfhtu3e6GjFpaD8GYhtuwAU44AebOhdNPh4cfhs6d/Y7K1FFgewgVk8qNcwL7R2BMw0UWo+vZE158EV56yZJBmgrst+Gywq0A5GYH9o/AmIb56CM4+mgoKHAF6J56Cs46y++oTAME9tuwWSOb4DKmXrZsgT/8Afr1gzVrYPVqvyMycRLYhLClpJxOrZr4HYYx6WX6dFeM7oEH4Kqr4Msv3foCkxECO6m8YsM2v0MwJv1MnAiNG8MHH7gegskogU0ILZvkkJ1lG28YU6uXXoI994S8vMpidE2sd52JAjtkVBoK07GVlb42plqrV7tJ4jPPdPsVALRoYckggwU2IazcuI1GdsmpMbtSdcXnevSA115zC8yeesrvqEwSBHbIKDcriw3FZX6HYUzqGT/elZro2xcefxwOOsjviEySBDYhhFXZd7cWfodhTGoIh916gn32gXPOcY8vuACyrBcdJIH9295eHqaprUUwBr7+2m1fGVmM7qKLLBkEUGD/xreWllvpaxNsZWVw552uGN2iRXDXXVaMLuACO2S0vTxMk9zA5kMTdIWFMHgwzJvnriR68EHo1MnvqIzPAvuNWBYK08jqGJmgqShG17Yt9O4Nkya5gnSWDAyBTghqhe1MsMya5RaXVRSje+IJOOMMv6MyKSSQ34ihsBIKW0IwAbF5M/zud9C/vxsq+uknvyMyKSqQ34il5WEAGtscgsl0U6e6fQpGjXIVSr/4Ao46yu+oTIpK2qSyiIwFegCvq+odNbQbBbypqlMSFUt52CWEHKtlZDLd5MnQrJkbLjruOL+jMSkuKQlBRM4AslX1WBF5QkT2V9Vvo7TrD3RKZDIAN2QEWHE7k5kmToS99oIjj3TF6LKzrf6QiUmyxkzygQne/beAXermikgu8BhQICJDExlMuZcQcmwOwWSSH390hejOPtvtVwDQvLklAxOzBn0jisjgGJs2B1Z69wuBjlHaXAwsAu4BjhKRq6s553ARmSMic9auXVvXkAEoD3kJwXoIJhOowpNPumJ0b7wBd9/tHhtTRzUmBBH5m4i0EJFOEce6iMgF3sMXYzzPFqCpd79FNec9HBijqquBZ4GB0d5IVceoap6q5u22224xnn5nFXMINmRkMsL48XDZZW4ns/nz4dpr3Z4FxtRRbT2EYbgv5hki0kxEsoAXgIqeQUmM55lL5TBRL6AgSpslwD7e/TxgWYzvXWdlXg8hN9sSgklToRAsWeLun3MOPPsszJgBBxzga1gmvdWWELYDbwKfA08CFwCNgSu852OtHz0ZuEhERgLnAAtFpOqVRmOBgSIyE7gSuC/G966zspB32WmO1TIyaWjRIremoH//ymJ0VpnUxEFt/UpV1XIRuQL32/sC4AfgTRH5JRDTr9iqWiQi+cAJwD3esND8Km02A2fXMf56qZhDsCEjk1bKytz8wN//7orQ/ec/VozOxFWsA433AtcAJwOTcF/mdfp1RFU3UHmlka8qLju1SWWTNgoLYeBAWLAAzj3XXUW0++5+R2UyTLVf6iLSHmjuXQ7aHZgKfAv8Dlji/UavEe3TZoNim1Q2aSOyGN1RR7mFZs8/b8nAJETUhCAiPYBVuERwAnAa0B43j/AWMExEQsAeIhISkTCwNSkRx4EtTDNpYeZM6NMHli51xegeewyGJnSJjgm46noI3wJDga+BO7xbETAPGAV8BRwG/OT9PAx3ZVBa2FxSDlhCMCmqqAiuvBJ+9jPYuBHWrfM7IhMQUROCqpap6lSgHBiAm1A+DHgJlyS64RaRlavqQlX9UlXnJSnmBquoclpR5M6YlPHmm64Y3aOPwp/+ZMXoTFLVOqmsqltE5GLgdmAF8CBueCg3wbElTMgbl23ZxBbvmBTz6qvuyqGPPoJjjvE7GhMwtX0jiohkA38HlnrHZgO9VLVURLT6l6ausDeHkCU2ZGR8pgoTJsDee7uewH33uVXGjdPmGg2TQWq7dLQJbmXxucBEr0Lpg8AE7+qjRgmOLyFsUtmkhJUr4Ze/hPPOg4cecseaN7dkYHxTW0J4GFgM9FfVNd6xe3GlJ1riitalnYohI+shGF+ouiuGevSA6dNdr8CK0ZkUUOOQkar+27v7fcSxMuBSABE5LWGRJVDYegjGT+PGwfDhkJ/vEsN++/kdkTFAAzfIUdX34hVIMlX0ECwhmKQJhdx6gv33dyuNc3PdvgVWf8ikkJgSgogcjKtsGsatTs7CFbkrUNXtiQsvMUI2qWyS6csv4fLLoaAAvvkGWrVyScGYFBNrD2EhEWUqIhwJfBa/cJJjQ3EpYD0Ek2ClpXDnnfCPf0Dr1q7+kBWjMyks1oRQBBzi3RfgS+/xmmpfkcKaNbL1BybB1q93cwRffgnDhsG//w313NDJmGSJ9ZsxrKoVW2AiIjs9TjfqdXYa59j4rYkzVVd3qF07OPZY+Oc/4dRT/Y7KmJgE8hux3K4yMonw3nvQu3dlMboxYywZmLRSY/lrEWktIi28x029bTSbV3nczNt3uV2ygm4oW6ls4mrTJvjNb2DQICgudnsXGJOGahoyWsvOE8lbqjwf+Vi8tmmxJ6WtVDZx89prMGIE/Pgj/OUvcNtt0KyZ31EZUy81JYSfAaW4iqfvAf294wLMjHgMrtBdm0QEmAg2ZGTi5s033XzByy/DkUf6HY0xDVJtQlDVDyrui0i5qs6v7nG6CdvCNFNfqm7Hsn33dcXo7r3XFaNrlJZlvYzZSSAnlUPeNgjZNodg6mLFCjjtNHcZ6ahR7lizZpYMTMaIOSFEm1ROXFiJFbI9lU1dhMMwerQrRvfOOzByJIwd63dUxsRdrOsQ2rDrJPIWETlSVdNupXJZyA0Z5WZbQjAxGD/eTRwPGuSK0e2zj98RGZMQsSaEvsB2dq5llIvbTjPthMJKdpYgNmRkqlNeDt99Bwce6PYraNwYzjrLrS8wJkPFlBBU9WMR6aCqGbHbd7mXEIyJasEC+PWvYfnyymJ0Z5/td1TGJFxdJpX/KyLXJCySJAqFw+RYQjBVbd8Ot9wCffrAsmVWjM4ETqzlrzsDJwIjReR53PoEgBDwqqq+nKD4EqIsZD0EU8X69TBgACxaBBddBP/6F7Rv73dUxiRVrD2EPwEfq+o7wGDgQ++2CnhORNJmURq4OQTrIRjArSsAt7hswAB4/XV4+mlLBiaQau0hiMj+wG+Bv3l1jTar6mjvuWxgENAM2JjIQOOpLBQmJzuQSzBMpHfegT/+ESZPdgvNHnnE74iM8VWN34oi0gp4DrgXuATYHWgiIgNFZBCuvMVfgYO8x2mhPKzkWg8huDZuhCuugMGD3bzBxrT5XcaYhKq2hyAiA4F/AXNU9XYRORu3jeZ64C7cJagVcoFGwGEJjDVuwmElyxJCML3yCvz2t/DTT3DddXDrrdA0bddYGhNXUROCiPQB3gEmqOrlkc+pas9kBJZIIbVJ5cCaPh123x2mTHFXExljdog6ZKSqc4GhQL6IDK84DCAiy0VkacRtmYjUunRTRMaKyMciclMt7TqKyLw6fo46CavthRAYqvDMMzB7tnt8zz3w6aeWDIyJoto5BFWdgpswvlVEzgOa4EpWZAGnA2cAi4BbgOU1nUREzgCyVfVYYB9voro69wEJ7cOHw4p1EALghx/glFPg4otdLSJwxehyc/2Ny5gUVeNVRqq6SEROAGYAFwM/AtsrSl+LSCHwraqW1XKefGCCd/8toB/wbdVG3sR0MbA69o9QdyFbqZzZwmF49FE3R6DqFphdeaXfURmT8mq99lJVFwE3ALfjho06ishLIvIS7ov+nyLSv4a3AGgOrPTuFwIdqzYQkUbAzcD1Nb2RiAwXkTkiMmft2rW1hR9VWNWGjDLZuHFw1VVuk/svv4Srr4bstNjMzxhfxVrLaKw3l/Br4M+4QncKvIIbSqrtN/otVA4DtSB6IroeGKWqG2sqOqeqY4AxAHl5eVptwxpYQshA5eWwZAkcdBCcf74bGjrjDCtGZ0wdxFrtFOBO4HpVPaYe55mLGyaaDfQiepXUwcAgEbkK6C0ij1e9wilebMgow8yfD5dd5jawWbLE1R8680y/ozIm7dRlue4U4OR6nmcycJGIjATOARaKyB2RDVR1gKrmq2o+8HmikgHAig3bbFI5E5SUwE03QV4erFzpdjGzYnTG1FvMPQRVDQEb6nMSVS0SkXzgBOAeVV0NVLsns5cUEqZjqyZ8+9PmRJ7CJNr69dCvHyxeDJdc4nYxa9fO76iMSWt1GTJqEFXdQOWVRr5SlK5tm/kdhqmPcBiystyX/8CBrirpSSf5HZUxGSGQFd5U3YIKk2beegt69nTzBCJuiMiSgTFxE9yEYBkhfRQWwq9+BUOGuB7CZhvuMyYRgpkQUMT6COnh5ZehRw9XfuLGG+Hzz+Hww/2OypiMlLQ5hFSiio0ZpYv33oMuXWDqVOjd2+9ojMlowUwIWD5IWapux7IDDnArje++G3JyrP6QMUkQyCEjbA4hNRUUuEniSy+Fxx5zx5o2tWRgTJIEMiHYHEKKCYfhwQfdFRXSRJ8AABOLSURBVEQffQQPPQSPP+53VMYETjCHjKyHkFrGjYPf/95dRTR6NOy1l98RGRNIwUwI2AY5visrg2++gUMOccXoWraEoUMtUxvjo0AOGYVV7XvHT599Bkcd5VYab97sJo1/+UtLBsb4LJAJQetVNNs02LZtcMMNLhmsXu02sbFidMakjMAOGdW054JJgPXr4bjj3DDRZZfBffdB27Z+R2WMiRDIhICqXWOULJHF6E48ER5+GAYP9jsqY0wUwRwywoark2LqVFd2oqIY3YMPWjIwJoUFMyFYtdPEWr/e7VFw8smud7Bli98RGWNiEMyEgNocQqJMnOh6BePHw803w7x5VoPImDQRyDkE6yEk0MyZ0K2b27ugVy+/ozHG1EFwE4JlhPhQhaeeggMPdFcR3X23qz2UE8h/WsaktYAOGdllp3Hx/ffuyqHLLoOxY92xpk0tGRiTpoKZEOyy04YJheA//3HF6D75BB55pLI6qTEmbQXyVzkbMmqgcePgj3+En//crTbu1s3viIwxcRDIhFAaClv567oqLXWrjHv2hGHD3CrjX/zCMqsxGSSQQ0aFxaUUl5b7HUb6mDMHjjwSBg2qLEZ36qmWDIzJMIFMCB1aNPI7hPSwbRtcey0cfTSsW+c2rbFidMZkrEAOGQG0amLbMtZo3Tq3p/GSJXDFFXDPPdCmjd9RGWMSKLAJwaYQqlFRjK59ezdpPHSoGyoyxmS8QA4Z2XYI1XjjDTj44MpidP/5jyUDYwIkkAkBrIOwk3Xr4MIL4ZRT3CrjrVv9jsgY44NgJgTrIlR64QXXK5gwAW691W1vedhhfkdljPFBIOcQrHRFhI8/hr33dqUnDj3U72iMMT5KWkIQkbFAD+B1Vb0jyvOtgeeBbKAYOFdVSxMWT6LeONWpustHe/SAvn3hrrvcMFF2tt+RGWN8lpQhIxE5A8hW1WOBfURk/yjNLgBGquqJwGrgpETFoxrQMaPvvoPjj4fhw12FUoAmTSwZGGOA5M0h5AMTvPtvAf2qNlDVUao63Xu4G/BTIgMK1IhRKAQjR7ohoblzYfRodzPGmAjJSgjNgZXe/UKgY3UNReRYoK2qzq7m+eEiMkdE5qxdu7ZewQSufzBuHPzf/7newcKFroeQFczrCYwx1UvWHMIWoKl3vwXVJCIRaQc8CJxZ3Rup6hhgDEBeXl69v9szvoNQWgpff+16BRdcULnQLFBdI2NMXSTr18S5VA4T9QIKqjYQkUbAi8ANqroskcFk/BTC//4HffpUFqPLznZrDCwZGGNqkKyEMBm4SERGAucAC0Wk6pVGvwaOAP4qIjNE5NxEBpSRl51u3eqGho49FjZsgCeftGJ0xpiYJWXISFWLRCQfOAG4R1VXA/OrtHkEeCQp8WTiLMK6da4q6dKlMGKEu5y0dWu/ozLGpJGkrUNQ1Q1UXmnkK9UMmkMIhdyQUPv2rhDd0KHws5/5HZUxJg0F91KTTMgIU6bAgQe6ncxE3KWllgyMMfUUyISQ9pPKa9fC+efDaadBs2awfbvfERljMkAgEwKQvnsqP/ecK0Y3aRLcfrvb3tJqEBlj4iCQxe3S2iefwP77u3pEhxzidzTGmAwS2ISQNledhsPw2GPuy79fPytGZ4xJmEAOGaVNcbtvv3WLy0aMgKefdsesGJ0xJkECmRAgxS8yKi+He+91G9V8/rkbHrJidMaYBAvkkFHK9w/GjYNrr3VrCkaNgi5d/I7IGBMAgUwIkIJzCNu3w+LF0KuX2994993hpJNSMFBjTKYK5JBRyk0hzJ4NRxwBgwdXFqM7+WRLBsaYpApmQkBTYx1CcTH86U9w3HEuETz9tBWjM8b4xoaM/LJuHRx1FHz/PVx1Fdx5pyUDY4yvApkQfB0yqihG16EDnHGGmzju39/HgIwxxgnkkBH41EOYPBkOOMAVowO47z5LBsaYlBHIhJD0DsKaNXDuuXD66W5YqLQ02REYY0ytApkQnCR1EcaNgx49XO/gH/+ATz+Fnj2Tc25jjKkDm0NItDlz3J4FY8e6KqXGGJOiApkQIIFzCOEwPPqoK0ndv78rRpeTY/WHjDEpL6BDRgnqInz9tdux7Kqr3FARQOPGlgyMMWkhoAkhzjMI5eWuJ9CrF3z5JTz5JDzySDzPYIwxCRfIhBD3OYRx4+CGG+CUU+Crr+DSS1Ng5ZsxxtRNIOcQlDh8X5eUuGJ0vXu7YnSdO8OJJ8YjPGOM8UUgewjQwD2VP/oIDj8cTjihshidJQNjTJoLZg+hvmNGW7bAjTfCQw9Bt27w7LNWf8iYWpSVlbFixQpKSkr8DiVQmjRpQteuXcnNzY35NYFMCFCPIaN16yAvD374wV1F9M9/WjIwJgYrVqygZcuWdO/eHbG5taRQVdavX8+KFSvYe++9Y35dIBNCnfoH5eVuHUGHDnD22a4YXb9+iQrNmIxTUlJiySDJRIT27duzdu3aOr0uwHMIMZg0Cfbbz60vALfPsSUDY+rMkkHy1efPPJAJodYphNWr4ayz3K1dO9dLMMakrcLCQlq2bLljHuPSSy9l1qxZAPztb3/j2WefJRQKMXz4cPr3788ll1xCOBxu0DnLyso49dRT6du3L0888US17T777DMGDx5M3759uf/++6s9BvDVV18xdOjQHY/nzZtH3759GTBgALfddluD4oWAJgSoIXs+/bSrOfTaa26x2f/+B4ccktzgjDFxNX36dEpKSpg5c2a1bV544QW2b9/OBx98QKdOnZg8eXLUdkOHDiU/P3/HbcyYMVHbPfjgg/Tp04cPP/yQiRMnsnnz5qjtrr76ap588klmzZrFpEmT+P7776Me++6777jmmmvYtGnTjtc++uijvPzyy8ycOZMJEybs9Fx9BHMOoaYuwuefu2qkjz/uitIZY+LmtikLWbSqKK7v2aNLK249teZf2qZOncpVV13F1KlTObGaS8SnTZvGKaecAsC5555LcXFx1HavvPJKTHHNmDGDu+66C4ABAwYwZ84cBg4cuEu7wsJCunXrBkD79u0pKiqKeqxz585MmjSJIUOG7Hjt6NGjAdcbKS8vp1mzZjHFVp1gJoTIB+EwPPywKzsxYIDbyjI3F7IC23kyJuN8/PHHzJo1i+OPP77aNmvWrKFdu3YAHHHEEQ0+Z3FxMXvssQcA7dq1Y82aNVHb9e3bl4ceeoh27dpRUFDAYYcdFvVYTXMCI0eOZNiwYXW6xDSapCUEERkL9ABeV9U76tsmfvHgykxcfrlbaPbb37qE0LhxIk9rTKDV9pt8IixYsIB169Zx1llnUVBQwPLly3f5chURWrVqxZYtWwCYPHkyW7Zs4cILL9zl/YYOHbrT0MywYcMYPnz4Lu1atGjBtm3baN26NVu2bKFFixZR4xs9ejTvvfcet9xyC9dddx0iEvVYdT755BPeeOMN3n777Zj+PGqSlF+DReQMIFtVjwX2EZH969MmXnLKy+k3YYwrO7F4sZs3ePjhRJ3OGOOjadOmceONNzJjxgx+//vfM23aNDp27MjSpUsBWLp0KZ06daJv375Mnz4dcHMObdq0ifp+r7zyCjNmzNhxi5YMAPr06bNj4nr+/Pl07949arvs7GwO9IanL7jggmqPRVNQUMCVV17J+PHjG9w7gORNKucDE7z7bwHRrt2MpU1c/PzL9zj+2QfcmoJFi+Cii6wYnTEZatq0aQwaNAiAQYMGMXXqVEaMGMHo0aPp378/JSUlDBw4kOHDh1NYWEi/fv0oKiri5z//eYPOe8kll3Drrbfyhz/8gUWLFnH00Ufz7rvv8tBDD+3S9qabbuLuu+/eqScQ7VhV1113HYWFhVxwwQXk5+fzdcUl8vUk9S7jUJeTuKGgB1R1voicCByhqnfVtY3XbjgwHGDPPffss2zZsjrHc8uk+Zy09iuOG3FefT6OMaYOvvrqKw4O6G6Bq1atYtasWQwZMoTWrVsn/fzR/uxFZK6q5kVrn6w5hC1AU+9+C6L3TGJpg6qOAcYA5OXl1Sub3X5mL6BXfV5qjDEx69KlC+ecc47fYcQsWUNGc6kcAuoFFNSzjTHGmARJVg9hMvCBiHQBTgbOE5E7VPWmGtock6TYjDEJpqpWviLJ6jMdkJQegqoW4SaNZwMDVXV+lWQQrU3DltwZY1JCkyZNWL9+ff3Lzps6q6h22qRJkzq9LmnrEFR1A5VXEdW7jTEmvXTt2pUVK1bUufKmaZiK/RDqIpArlY0xyZObm1unmvzGP1afwRhjDGAJwRhjjMcSgjHGGCBJK5UTRUTWAnVfqux0ANbFMZx0YJ858wXt84J95rraS1V3i/ZEWieEhhCROdUt385U9pkzX9A+L9hnjicbMjLGGANYQjDGGOMJckKIvhFqZrPPnPmC9nnBPnPcBHYOwRhjzM6C3EMwxhgTIeMTgoiMFZGPReSmhrRJJ7V9HhFpLSJvishbIvKyiDRKdozxFOvfn4h0FJF5yYorkerwmUeJyKnJiiuRYvh33VZE3hCROSIyOtnxJYr37/aDGp7PFZEpIvKhiFzWkHNldEJItb2ckyHGz3MBMFJVTwRWAyclM8Z4quPf331UbsKUtmL9zCLSH+ikqlOSGmACxPiZLwLGeZdjthSRtL8UVUTaAv8FmtfQ7Gpgrqr2Bc4SkZb1PV9GJwRSbC/nJMmnls+jqqNUdbr3cDfgp+SElhD5xPD3JyKDgGJcAkx3+dTymUUkF3gMKBCRockLLWHyqf3veT3QU0TaAN2A5ckJLaFCwLlAUQ1t8qn8s5kJ1DsRZnpCaA6s9O4XAh3r2SadxPx5RORYoK2qzk5GYAlS6+f1hsRuBq5PYlyJFMvf8cXAIuAe4CgRuTpJsSVKLJ95FrAX8HvgK69dWlPVohj2honbd1imJ4S47eWcRmL6PCLSDngQaNCYYwqI5fNeD4xS1Y1JiyqxYvnMhwNjVHU18CwwMEmxJUosn/lWYISq3g4sBn6VpNj8FrfvsHT/8qtNEPdyrvXzeL8xvwjcoKr1rQWVKmL5+xsMXCUiM4DeIvJ4ckJLmFg+8xJgH+9+HvWv+ZUqYvnMbYFDRSQbOBoIyjX18fsOU9WMvQGtgPnASFwXshdwRy1tWvsddxI+82+BDcAM73au33En8vNWaT/D75iT9HfcEpf0ZwIfA3v4HXcSPvNRwELcb8zTgRZ+xx3Hzz/D+zkI+F2V5/byPvd/gE9xk+/1Ok/GL0zzZulPAGaq6z7Xq006ybTPU5ugfV6wzxyUzxwrEemC6yVM0wbsR5/xCcEYY0xsMn0OwRhjTIwsIZjAE5EcETkzwedoKiLNamnTSEQOSWQcxtTEEoIxsCfwqIjcCCAifUWkXEQ2VrkVi8gr0d5ARLaIyC9qOMet1F6hchAwT0Q61OtTGNNAOX4HYIzfVHWpiJwAzBKRWUAJ8LlW2ZFKRM4DTo94PBl4QlVfxa2CLvJ6AR8A5wHbgMaq+h2w3btVvPZPwFXesYqJvGwgF/hURIormgKNgfGqekt8P7kxO7OEYAygqp+LSD/v55Exvuwz4A7gVaAUCOMWwa1W1W9F5HbgIOAc3Jd+5BUcj+EuCV2gqmVe/ZkPgNOAt1R1O+woQXEy7vJgYxLKEoIJNBHJArJUtVxVP/cO5+IWOBVUad4c+DDi8WPAKhHJwSUDcNe/j/ful+J6G9HsjkskE7zqnU8Cb+Pq9LwiIj8Af8ENMx2Bq1dTUz0bYxrMEoIJuiNwQzTgKmVeqKof4YZpanM1rohaf6A9cCOwDlhFZd2kcLQXesNUecBZwO24nsEC71YIXINbaZwN9FHVzfX6dMbUgSUEE3RrgHtxpQ4qhmkaAyFVLY9sKC5rZOPW75Thhnia4770T8YVV/sG2BrLiVX1RxGZhpsneAM3P9EHuBS3yOhNYEqs72dMQ9lVRibQVHW5ql4LvEflGP+rwHIRWeFdXVQiIitwFSVXAHd5r31TVSeq6ku4L+2Z3uM3ajuvd4npjbiksh2XUPbGTUb3AyYBA3C9kPdFpGscP7YxUVkPwZgI3pzCaUCpqqo3MXyIqp7pPZ8N5IjIEOByoAzXQ+gA3Cwi64BGwAO1nOox3JBQb+BKYASufn8B8Bxu6OhmoK/XdiDwTPw+qTG7soRgzM76A1OBUhFRXFE1EZGK0tm5uC/8P+I2JSnHlRy+AHgBVzQwB/gBN7yUXc15/gxsVtVSEbkDt06hB/CFqg4TkVZAWFVLgUvi/zGN2ZUlBGMqZQEHqGpTABE5APgCN1F8tTc0tAsRORlYqapPVDneFGgSpf0g3LDUVhEpjXgqx3t+RUTbiuNNgbNV9a16fzpjamEJwRgnGzd+30dEnsMNAz0HjMVdDjpKRBar6qLIF3lDTLcQZRWyqkbdoU1V38X1KnYiIj1xPQSbLzC+sEllY5whwHe4idxuwEe4yd5rvJ7BI8CHIvIXrwwz3mKyF3G/vd8T7U29OkkCNKP2DVvEe400/OMYU3eWEIxxrsRd9nknbpjoI+BEVS0GUNXbcJeDjsBdgdQNtylJN+AEVa1uAVoXvMTivW9Ncr2fuwwzGZMMth+CMRFEZD/cGoTvq3k+GzhQVRd5Qzxfe2sSanrPQcAKVf0m/hEbEz+WEIwxxgA2ZGSMMcZjCcEYYwxgCcEYY4zHEoIxxhjAEoIxxhiPJQRjjDGAJQRjjDGe/wfJtRFwAtgMsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.99\n"
     ]
    }
   ],
   "source": [
    "prob = clf.predict_proba(X)[:, 1]  #获取概率\n",
    "auc_plot(y,prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAEYCAYAAACeIFADAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAYhklEQVR4nO3dd5hV1bnH8e8rjPSqIGJBFKxo6CBSFUQ0iUpVMQajDJZrojHJjS03JiKYmKgIChjsGizXRGOUQFSkinSxYCFSggooA8ioV2Te+8faA8dxZjjEc+YMi9/neeaZs8tZ+x2G31lr1zF3R0TitU+uCxCR7FLIRSKnkItETiEXiZxCLhI5hbwSMbOqZlalArf3P2Z2fkVtT3LDdAqt8jCzy4F8YBuwP1AHeD9llWOBbu6+MFn/KeAEYGsZTTYEnnD3q5P1JwEtgaJk+SFATeDtZLoK8LG7n51GrfWB14Ce7v6vZN5ZwBigEBjq7ovSaKc58DJwvLtvNrMBSRsD3X2umQ1x98d21Y6UrWquC5Cd3H0cMA7AzO4ANrj7TcXLzWwt8EHKW7YBF7v79NLaM7NhwNEps0YCn7n7R2Z2JPAnoA+wj7t/bmbHAdvTLPcm4M8pAW+ctNcD+AJ4DGifRjtjgFHuvjmZ/kHyNdjM3kWjzW9N/4CVkJk1AgYAD6XMywPquPuHKatuS5a1NrMFJb5qJOt8lbL+ScAUMzscuB24ljByuN/MWgKPA22SNp82s49KfN2cLGsD9Cd8aBTrC0xz9zfcfQWwOvkgKe/n/D7QHJiYMrsusDr53h/46y7+uWQX1JNXMmbWHXgAuMndVyXzzgYGAXPKeFsesNzdz0/WX8nXww2Auz9kZu8A3YCmwB+ADcAQQo+f7+6zk3XPLKfMO4AtwMNm9jwwHjiYMHwvtgpoAbxTxs+ZB9wGbAL+18z+nAzLNyXv+xTIc/fPy6lD0qCevJIws+Zm9howCrjE3cenLF4PvAgMLePtpR1Y+cY8M6tDCPTbhJ58CaEnfwmoDow2s+N3UWcXoAswAXgQuBoYQdif35KyaiFQv5ymBgEHEHZPngTGmtlpwL3JvAKghpktN7PDyqtJyqeevJJw9/eTHnsAcIeZlbbaz8zsOHcvud9cHTjdzJYk002BaqkrJEftZwJ3EQ6+XQ18BtxPOLh3NvBz4GLgJ2b2NNCpxHbuJfT8f3P325J2awGDgef4eqhrsPMAX2k6Afe4+71JO0cA33P3y4FnzewnQE/gVuB7wJ3ltCXlUMgrEXdfYWb1gPvdfXTJ5Wa2uZSA4+6zCEfSS67/tUmgl7sXJNMPmdlNwEeEXrnI3beb2UIzq1rWcN3MLiLsMxf7gjDEXgDckDK/PfCXsn9atgAbS2mnOPDvAWcSPoC6lNOO7IJCXvmUd06z5LJSu/sy1jkD+KWZ/V/KsubAl8BA2PGhUIUwXJ5cRnuzgF8kBwe3Enr+J4BXgZZmdjFhN7BFMq8sM4Bfm9lEwqjjfMLoAqAfYcSRT9jX31JqC5IWhbxy+rGZnVPK/FolpvOAu8zsszLa2Y9wxBx3fxp4OnVhcU/u7mPTLczd3zazkcDspP2/AZPc3c3sTMIpsbrAAHf/0swOAf7u7ieUaGeambUGXk/Wn+juU5Pdii3uXmRmk4HRQO9065Nv0sUwlUwSvC3u/rtSlm0F6rv7V8n0HwgBe7OMtr4PtHT3P5Sx/BZCyG/L2A8glY5CLhI5nULbS5iZds32Ugr5HsLMJplZnxLzLjGzX6XZxA1mdmua29KNKxHRcL2SS64nbwZ8n3DgrCvhAFxboDGwL7AG2OruZyTveR8oPor+f4Rz70sJV59tBla4+0Up28jIjSvJB05+MnmDu9+XzL8E+DXwb2CQu79fegtgZvsCDyc/70eEa/P/mRx/6Eu4Wm8z0N/dnyyvHgk0hKv8CghXhM0nXK32CXAL4S617ybf7yOcZy7WgnBU/VZCUF8Ehrn7E2Y2DphaYhvf+sYVM2tFuDmlGeFinIVm9hzhg+gGwodSM2As4XReWa4mXJPfhBD0uwkfQO2BSUB3wiW7i8urR3bScL3y+4IQ6trA54S7uw4gXCjSlnD76ZnAUSnvySME/RVCj/w8cLmZTQeGA1eZ2XQza5asn4kbV44BFrj7dndfQ7h2vQlwFvCgu3/g7nOB/ZOr5MqyFLjM3TcBzxLOk0O4RmAz4XRbi+QmGEmDevLK70hC73UaoUfuCJxHuBBlf8JwvTvQz8xqufvjwEWEy0onEkYCfyUEfyxwq7sPNLMpJL//TNy4kvT2vzGz2wkfMPsDbwGXAdNTVv2A0KOXetrP3Z9LmexN+KCC0CE1JIwo3i75PimbevJKzt1fIfSuNYB/AE8RrjKbCjxKuPZ8GvALd388CdtvgWcIw+RWhN/zEMJVbD2SHr0jyRA8EzeuEEK7Bvg74Rr3W939S3b/xhWSmqoBNxJGMRAeLHEOYWTSy8xesAp8is6eTCHfM1xM6AEPIYS6NaGHX5B87Q+cnKxbjbCPXqyI0Ns/RgjJy+7ek+SS05QbV5az88aV1pR+40p5w/VhwPvu3paw63CWmXUgjCR258aVYr8Blrj7FAB3v4Ew0lgPdCB8qBybRjt7PQ3XKzkzawGcSrjL6zlCaGoShrLFD2VoQggl7r7IzOoSDsoVq5OsfyDQxszGszMgmbpxZQLJEDy5JHUJ4WDZAkI4H7FwcXxbYO0ufuZ+hFtR25ZYdDbhppfz+OaHh5RBIa/82hF6te4A7j7KzAYSrvUeDTse89SkxPvOIYRrI+Gg2bOEYX1DwlH34pBn6saV94Ehya5AA+Dc5Gsp8Ecze5Gw6/BJcmCuVMmH2oPAGcnBt1T7u/vHST110Y0raVHIK7nihxiaWY+U2VWA/OQhC5DSk6csnwzcTNg/H0DY5/4MKHT398zs86T9jNy4QvgAaEe4u2wbcLe7z0zaGwj8nnDOfmgyrwtwXfG5/RSXA/WAZ1JulW1D+OCYl0y/Qbgf/We7Ud9eSyHfc+ybfBW/nujuo83sKMJR9NRHL60Ennf3jcmdXJ+4+5sWnrD6pJkNJgx1PyljW3nJV9rc/VPCELu0ZXMJF/GkzptDKefL3f0q4KpSmvkwZZ3/2p3a9na64m0PkRxt/qrkQyOS/dwqxXemiZSkkItETqfQRCKnkItELqcH3qxqDbd96+SyBElDm2MOzXUJkoZFixZ+7O6NSs7Pbcj3rUO1owbnsgRJw+x5u3MmTXKlRp6tKm2+husikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhK5qrkuoLJr1nQ/bvvlIOrWqs6CN1axYs0GBp7aDoB6dWowf9lKbhjzNPfdPIxGDeuw+M3VXDFy8o73337NYKbOfpPnZrzO8EFdv/He1HUlszZv3swFQ89h+/bt1KxVi+7de/LM038JyzZtokPHTvz25tFceMFQNqxfT5u27Rh794QcV5156sl3YeRPzmT0PVPofdHtHNS4Psv/tY6+w++g7/A7mLN4Bff9ZQ7nfbcjjz03n65Df0edWtVpe+yhAJzU5gia7FeX52a8DsA9T8z6xnsleyY/+gg/vvKnPPv8VJoc0ITDmjdn6gvTmfrCdLp07caFFw3n0YcfYsi5Q5k9bwGfbv2UhQsW5LrsjFPId6FFs8YsfmsNABsKtlKvTnUAmjaqR+OGdVj05mo2birk2BZNqVe7BgcfUJ9/f1RA1ar7MO6Gc1n14Ua+2/P4r7WZ+l7JnhGXXsYpvfsAsOHjDTRq1BiAtWvXsn79Otq1b0/D/fbjzTdeZ9OmTfx7zRoOPuSQXJacFQr5Lvzln4u5bsTpnN69FX26HMNL894GYMSQ7kx8YiYAc5as4NADG3LZeT1Y/v46Nm4pZOh3O7H8Xx/xx/un0f64w7j0nB472kx9r2TfK3PnsqmggE6dOwMw4e5x5I+4FIAuJ3Vl9epV3DV2DEcffQwNGzbMZalZkZWQm9kkM5trZtdno/2KdMuf/sHU2W8w7OwuPPy3eRR+/iVmRo8ORzJjwbsAXJt/OleMnMyoiVN4Z+U6Lvj+ibQ+6mAmPTWbdZ98yp+fe5Ue7VsCfOO9kl0bN27k6iuvYPw99wJQVFTEy9NfonuPngDc/NsbuXPceK69/lccedTRPHj/fTmsNjsyHnIz6w9UcfcTgcPNrGWmt1HRlr79bw5p0oAxD78IwEltj2D+spU7ljeoW5NWLZqyzz5Gh1bNcHdWrNlA84P2B6DtsYey+sONpb5XsufLL79k6DmD+M3IUTRr1gyA2bNm0qFjJ8wMgIJNBby+bBnbt29n/vx5O+bHJBtH13sCjyevpwJdgR3dlpnlA/kA5NXOwuYz76of9mbMwy/y+RfbAOhz4jHMWvTejuW/v3cqE248n0MPbMi8197n8SkLMDMm/Hoog05rR17VKpz3sz+V+l7JnvvvncSSxYu4ZdRIbhk1kuEjLmXZa0vp2q37jnV+/otrGDH8QlavWkWnzicy+Jxzc1hxdpi7Z7ZBs0nAGHdfamanAm3dfXRp6+5Ts7FXO2pwRrcvmVcwf2yuS5A01Mizhe7evuT8bOyTbwVqJK9rZ2kbIpKmbARwIWGIDvAdYGUWtiEiacrGPvlfgZlm1hToB3TOwjZEJE0Z78ndfQvh4NsrQC9335zpbYhI+rJy7bq7F7DzCLuI5JAOiolETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItETiEXiZxCLhI5hVwkcgq5SOQUcpHIKeQikVPIRSKnkItErsy/oGJmLwEl/66xAe7uJ2e1KhHJmDJD7u69KrIQEckODddFIrfLP3hoZgacARwAvAmscvcPsl2YiGRGOj35Y0AvYESy/sNZrUhEMiqdkDdy96uBre4+O833iEglkU5g3zWze4GmZvY/wDtZrklEMmiX++Tunm9mZwLLk6/fZL0qEcmYXfbkyYE3B7YBVQjnykVkD5HOcP0RoB9QCJwCTM5qRSKSUbscrgNN3f284gkzm569ckQk08q7rLV78nKzmV0PzAPaEXp0EdlDlNeTF1/WuoiwL94lmX41qxWJSEaVd+36jcWvzawRUCOZPCjbRYlI5qRzWeskoDnQAPiMcKS9a5brEpEMSefoegvgNOA9oAdQlNWKRCSj0gn5Z4RTZ1WAQYQeXUT2EOmEfCDwLnAVcAxwWVYrEpGMSuey1kLCUB3gV9ktR0QyTXeUiUROz3gTiVxOn/HW+phDmf3KndnejHxLRUUlP+tlT6LhukjkFHKRyCnkIpFTyEUip5CLRE4hF4lcOk+GwcxaEW4xXQ2scfetWa1KRDImnQc53gncCIwCDgcezXZRIpI56QzXj3f3AcAmd/87UC/LNYlIBqUT8g1m9iuggZn9EPgoyzWJSAalE/ILgM3AXEIvPiybBYlIZqUT8kFAAeFprZuSaRHZQ6QTcku+agD9ge7lry4ilUk6D414IGVyvJndlcV6RCTD0nlaa2rP3Qg4NnvliEimpXMxTOp95V8Cl2epFhHJgnSG6zfuah0RqbzSueLt+YooRESyI52j68vM7MysVyIiWZHOPnkH4AozW0b4i6Z6kKPIHiSdffKsP9BRRLKnzOG6hugicShvn/wnFVaFiGRNecP1zmb2Tol5xX9c4cgs1iQiGVReyOdpf1xkz1fecP3JCqtCRLKmzJC7+7iKLEREskNPaxWJnEIuEjmFXCRyCrlI5BRykcgp5CKRU8hFIqeQi0ROIReJnEIuEjmFXCRyCrlI5BRykcgp5CKRU8hFIqeQi0ROIReJnEIuEjmFPEM2btzIC/+cxscff5zrUkS+RiH/D0yccDd9e/eib+9edGrfhv+6bAQDzvoeC+a/Sr8+J7NhwwYALsm/iJ7dujD65ptyXPHeZ926dfQ5ufuO6eVvvcXgAWftmF6yZDGn9OzKqaf04Oabwh/unTN7Fscf05LT+vTi9L69K7zmbEnnb6FJCfkjLiV/xKUA/PTKKzi7/0CqVatGx06d2bSpgCWLF1FYWEjR9iKmz5zDiOE/4r1336VFy5Y5rnzvUFBQQP5FwygsLATgXytWcN01v6CwcOuOdSZNHM+fH3+Kxo0b0751Ky6/4koWzH+VW28bQ9/T+uWo8uxQT/4trF27lvXr1tOtew86durMrJkzWDB/Pp06n8jMGdPpP3AQAL17n8qcObNyXO3eo0qVKjz4yGTq1q0LQO06dXj0sa8/YfzOuybQuHFjtm3bxldffUXNmjWZ/+o8bhl1Eyd2bMv4u8bmovSsyErIzewAM5uZjbYrkwl3j2P4iEsAcHeefOIx6tdvQF5eHoWFhTRtehAADRo2ZP26dbksda9St25d6tWrt2O6cePGVKtWrdR1x9zxRwafcy55eXmcf8Ewpr7wMi/NmMOfJo7nk08+qaiSsyrjITezBsADQK1Mt12ZFBUVMePl6XTv0RMAM+P2MeNodfzx/P1vz1C7dm2++OJzALZu3UpRUVEOq5XSzH91HlOnPM/P//taAHr2OpmqVatSvXp1Wh55FKtXrcxtgRmSjZ58OzAE2JKFtiuN2bNm0qFjR8yMP/z+Fh556EEANm/eRL369WnTph1zZoch+rLXltKs2WE5rFZKWrVyJVf++HLue+AR8vLycHd69+rG1q1bWbduHUuXLObwI1rkusyMyPiBN3ffAqFnK42Z5QP5AIccemimN19hpk39Byd1DUdvf3RxPj84bwj33zeJY487jt59TuXTTz+lT6/ufPjhB0ydMoXps+bmuGJJdcN1v6Rg40Z+NOx8AMaMHc+Pr/wpXTq2pVbt2owc/buvDfn3ZObu2WnYbLq79yxvnbbt2vvsV+ZnZfuVQUFBAS/+cxondetOkyZNcl3OfyxL/0Ukw2pV22ehu7cvOV+n0LKoQYMGDBg0ONdlyF5Op9BEIpe1kO9qqC4iFUM9uUjkFHKRyCnkIpFTyEUip5CLRE4hF4mcQi4SOYVcJHIKuUjkFHKRyCnkIpFTyEUip5CLRE4hF4mcQi4SOYVcJHIKuUjkFHKRyCnkIpFTyEUip5CLRE4hF4mcQi4SOYVcJHIKuUjkFHKRyCnkIpFTyEUip5CLRE4hF4mcQi4SOYVcJHIKuUjkFHKRyCnkIpFTyEUip5CLRE4hF4mcQi4SOYVcJHIKuUjkFHKRyCnkIpFTyEUip5CLRE4hF4mcQi4SOYVcJHLm7rnbuNkGYFXOCsiO/YGPc12E7FKMv6dm7t6o5MychjxGZrbA3dvnug4p3970e9JwXSRyCrlI5BTyzJuY6wIkLXvN70n75CKRU08uEjmFXCRyCnkGmdkkM5trZtfnuhYpm5kdYGYzc11HRVHIM8TM+gNV3P1E4HAza5nrmuSbzKwB8ABQK9e1VBSFPHN6Ao8nr6cCXXNXipRjOzAE2JLrQiqKQp45tYC1yeuNwAE5rEXK4O5b3H1zruuoSAp55mwFaiSva6N/W6kk9B8xcxayc4j+HWBl7koR2alqrguIyF+BmWbWFOgHdM5xPSKArnjLqOTIbR9ghrt/lOt6REAhF4me9slFIqeQi0ROIReJnEJeiZjZr83sLTObYWYvJEfq/9N2eqZM357Ge1qbWev/dBtlrDP927axO+1J6RTyymeku3cH7gOuyESD7n5lGqu1Tr4kMjpPXnk1AD6HHT3YfOAEd+9rZjWBB4HGwDJ3vzw5ffcEUAUwYHpxQ2Y23d17Jq+rA/cDBwObgMHADcDZyfIfuPspu7uNXTGz2sCThMt/33P3C5NFPzWzG4EPgaFAEeGpLUcCG4Ah7r493e3IN6knr3yuM7MZhItp7kjmdQbmunvfZDofeD3p8Q80sxOSec+6ey9gWznt5wNL3b0r8L9AK3e/BhgNjHb3UzKwjdIcCNwJ9AYOM7Pia/sXuHsPYDPwPeBMIC+Ztxo4Yze3IyWoJ698Rrr7wyXmve7uT6VMHwV0SfZn6wMHAc2Bx5LlC8pp/2hCuCH06GX5NtsozTbgYuBCoCE7r/Ofl3xfBBxB+D95YjJ6qQ28tZvbkRLUk+8ZtpaYfhu4PRmCX0/o8VYDxyXLy9u3Xg50SF5fSwgehF2DmgBmZt9yG6W5iDBcPxcoTJnfLvl+AuF6/7eBycl2rwTe3M3tSAkK+Z7pHqBfMqy/BFhD2I8dkPSAdXfx3rbJem2Bh5L504D+ZjYb6PYtt1GaacA1wIvJ9EHJ925m9jLh1tyngWeApsm8m4jvL+xUOF3WKhI59eQikVPIRSKnkItETiEXiZxCLhI5hVwkcv8PU41D17ArD2oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics_plot(y, prob, thres=0.45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
